{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "# 天池o2o优惠券使用预测比赛解析（初级）"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**赛题链接：**\n",
    "\n",
    "[天池o2o优惠券使用预测](https://tianchi.aliyun.com/getStart/introduction.htm?spm=5176.100066.0.0.518433afBqXIKM&raceId=231593)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 导入相关库"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# import libraries necessary for this project\n",
    "import os, sys, pickle\n",
    " \n",
    "import numpy as np\n",
    "import pandas as pd\n",
    " \n",
    "from datetime import date\n",
    " \n",
    "from sklearn.model_selection import KFold, train_test_split, StratifiedKFold, cross_val_score, GridSearchCV\n",
    "from sklearn.pipeline import Pipeline\n",
    "from sklearn.linear_model import SGDClassifier, LogisticRegression\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "from sklearn.metrics import log_loss, roc_auc_score, auc, roc_curve\n",
    "from sklearn.preprocessing import MinMaxScaler\n",
    "from xgboost import XGBClassifier\n",
    "# display for this notebook\n",
    "%matplotlib inline\n",
    "%config InlineBackend.figure_format = 'retina'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 导入数据"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>User_id</th>\n",
       "      <th>Merchant_id</th>\n",
       "      <th>Coupon_id</th>\n",
       "      <th>Discount_rate</th>\n",
       "      <th>Distance</th>\n",
       "      <th>Date_received</th>\n",
       "      <th>Date</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1439408</td>\n",
       "      <td>2632</td>\n",
       "      <td>null</td>\n",
       "      <td>null</td>\n",
       "      <td>0</td>\n",
       "      <td>null</td>\n",
       "      <td>20160217</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1439408</td>\n",
       "      <td>4663</td>\n",
       "      <td>11002</td>\n",
       "      <td>150:20</td>\n",
       "      <td>1</td>\n",
       "      <td>20160528</td>\n",
       "      <td>null</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1439408</td>\n",
       "      <td>2632</td>\n",
       "      <td>8591</td>\n",
       "      <td>20:1</td>\n",
       "      <td>0</td>\n",
       "      <td>20160217</td>\n",
       "      <td>null</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1439408</td>\n",
       "      <td>2632</td>\n",
       "      <td>1078</td>\n",
       "      <td>20:1</td>\n",
       "      <td>0</td>\n",
       "      <td>20160319</td>\n",
       "      <td>null</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1439408</td>\n",
       "      <td>2632</td>\n",
       "      <td>8591</td>\n",
       "      <td>20:1</td>\n",
       "      <td>0</td>\n",
       "      <td>20160613</td>\n",
       "      <td>null</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   User_id  Merchant_id Coupon_id Discount_rate Distance Date_received  \\\n",
       "0  1439408         2632      null          null        0          null   \n",
       "1  1439408         4663     11002        150:20        1      20160528   \n",
       "2  1439408         2632      8591          20:1        0      20160217   \n",
       "3  1439408         2632      1078          20:1        0      20160319   \n",
       "4  1439408         2632      8591          20:1        0      20160613   \n",
       "\n",
       "       Date  \n",
       "0  20160217  \n",
       "1      null  \n",
       "2      null  \n",
       "3      null  \n",
       "4      null  "
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dfoff = pd.read_csv('data/ccf_offline_stage1_train.csv')\n",
    "dfon = pd.read_csv('data/ccf_online_stage1_train.csv')\n",
    "dftest = pd.read_csv('data/ccf_offline_stage1_test_revised.csv')\n",
    " \n",
    "dfoff.head(5)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 简单统计"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "简单统计一下用户使用优惠券的情况："
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "有优惠卷，购买商品：75382\n",
      "有优惠卷，未购商品：977900\n",
      "无优惠卷，购买商品：701602\n",
      "无优惠卷，未购商品：0\n"
     ]
    }
   ],
   "source": [
    "print('有优惠卷，购买商品：%d' % dfoff[(dfoff['Date_received'] != 'null') & (dfoff['Date'] != 'null')].shape[0])\n",
    "print('有优惠卷，未购商品：%d' % dfoff[(dfoff['Date_received'] != 'null') & (dfoff['Date'] == 'null')].shape[0])\n",
    "print('无优惠卷，购买商品：%d' % dfoff[(dfoff['Date_received'] == 'null') & (dfoff['Date'] != 'null')].shape[0])\n",
    "print('无优惠卷，未购商品：%d' % dfoff[(dfoff['Date_received'] == 'null') & (dfoff['Date'] == 'null')].shape[0])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "可见，很多人（701602）购买商品却没有使用优惠券，也有很多人（977900）有优惠券但却没有使用，真正使用优惠券购买商品的人（75382）很少！所以，这个比赛的意义就是把优惠券送给真正可能会购买商品的人。"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 特征提取"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>User_id</th>\n",
       "      <th>Merchant_id</th>\n",
       "      <th>Coupon_id</th>\n",
       "      <th>Discount_rate</th>\n",
       "      <th>Distance</th>\n",
       "      <th>Date_received</th>\n",
       "      <th>Date</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1439408</td>\n",
       "      <td>2632</td>\n",
       "      <td>null</td>\n",
       "      <td>null</td>\n",
       "      <td>0</td>\n",
       "      <td>null</td>\n",
       "      <td>20160217</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1439408</td>\n",
       "      <td>4663</td>\n",
       "      <td>11002</td>\n",
       "      <td>150:20</td>\n",
       "      <td>1</td>\n",
       "      <td>20160528</td>\n",
       "      <td>null</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1439408</td>\n",
       "      <td>2632</td>\n",
       "      <td>8591</td>\n",
       "      <td>20:1</td>\n",
       "      <td>0</td>\n",
       "      <td>20160217</td>\n",
       "      <td>null</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1439408</td>\n",
       "      <td>2632</td>\n",
       "      <td>1078</td>\n",
       "      <td>20:1</td>\n",
       "      <td>0</td>\n",
       "      <td>20160319</td>\n",
       "      <td>null</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1439408</td>\n",
       "      <td>2632</td>\n",
       "      <td>8591</td>\n",
       "      <td>20:1</td>\n",
       "      <td>0</td>\n",
       "      <td>20160613</td>\n",
       "      <td>null</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   User_id  Merchant_id Coupon_id Discount_rate Distance Date_received  \\\n",
       "0  1439408         2632      null          null        0          null   \n",
       "1  1439408         4663     11002        150:20        1      20160528   \n",
       "2  1439408         2632      8591          20:1        0      20160217   \n",
       "3  1439408         2632      1078          20:1        0      20160319   \n",
       "4  1439408         2632      8591          20:1        0      20160613   \n",
       "\n",
       "       Date  \n",
       "0  20160217  \n",
       "1      null  \n",
       "2      null  \n",
       "3      null  \n",
       "4      null  "
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dfoff.head(5)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 打折率 Discount_rate"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Discount_rate 类型：\n",
      " ['null' '150:20' '20:1' '200:20' '30:5' '50:10' '10:5' '100:10' '200:30'\n",
      " '20:5' '30:10' '50:5' '150:10' '100:30' '200:50' '100:50' '300:30' '50:20'\n",
      " '0.9' '10:1' '30:1' '0.95' '100:5' '5:1' '100:20' '0.8' '50:1' '200:10'\n",
      " '300:20' '100:1' '150:30' '300:50' '20:10' '0.85' '0.6' '150:50' '0.75'\n",
      " '0.5' '200:5' '0.7' '30:20' '300:10' '0.2' '50:30' '200:100' '150:5']\n"
     ]
    }
   ],
   "source": [
    "print('Discount_rate 类型：\\n',dfoff['Discount_rate'].unique())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "打折率分为 3 种情况：\n",
    "\n",
    "- 'null' 表示没有打折\n",
    "\n",
    "- [0,1] 表示折扣率\n",
    "\n",
    "- x:y 表示满x减y"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**处理方式：**\n",
    "\n",
    "- 打折类型：getDiscountType()\n",
    "\n",
    "- 折扣率：convertRate()\n",
    "\n",
    "- 满多少：getDiscountMan()\n",
    "\n",
    "- 减多少：getDiscountJian()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Convert Discount_rate and Distance\n",
    "def getDiscountType(row):\n",
    "    if row == 'null':\n",
    "        return 'null'\n",
    "    elif ':' in row:\n",
    "        return 1\n",
    "    else:\n",
    "        return 0\n",
    "\n",
    "def convertRate(row):\n",
    "    \"\"\"Convert discount to rate\"\"\"\n",
    "    if row == 'null':\n",
    "        return 1.0\n",
    "    elif ':' in row:\n",
    "        rows = row.split(':')\n",
    "        return 1.0 - float(rows[1])/float(rows[0])\n",
    "    else:\n",
    "        return float(row)\n",
    "    \n",
    "def getDiscountMan(row):\n",
    "    if ':' in row:\n",
    "        rows = row.split(':')\n",
    "        return int(rows[0])\n",
    "    else:\n",
    "        return 0\n",
    "\n",
    "def getDiscountJian(row):\n",
    "    if ':' in row:\n",
    "        rows = row.split(':')\n",
    "        return int(rows[1])\n",
    "    else:\n",
    "        return 0\n",
    "    \n",
    "def processData(df):\n",
    "    \n",
    "    # convert discount_rate\n",
    "    df['discount_type'] = df['Discount_rate'].apply(getDiscountType)\n",
    "    df['discount_rate'] = df['Discount_rate'].apply(convertRate)\n",
    "    df['discount_man'] = df['Discount_rate'].apply(getDiscountMan)\n",
    "    df['discount_jian'] = df['Discount_rate'].apply(getDiscountJian)\n",
    "    \n",
    "    print(df['discount_rate'].unique())\n",
    "    \n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[ 1.          0.86666667  0.95        0.9         0.83333333  0.8         0.5\n",
      "  0.85        0.75        0.66666667  0.93333333  0.7         0.6\n",
      "  0.96666667  0.98        0.99        0.975       0.33333333  0.2         0.4       ]\n",
      "[ 0.83333333  0.9         0.96666667  0.8         0.95        0.75        0.98\n",
      "  0.5         0.86666667  0.6         0.66666667  0.7         0.85\n",
      "  0.33333333  0.94        0.93333333  0.975       0.99      ]\n"
     ]
    }
   ],
   "source": [
    "dfoff = processData(dfoff)\n",
    "dftest = processData(dftest)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>User_id</th>\n",
       "      <th>Merchant_id</th>\n",
       "      <th>Coupon_id</th>\n",
       "      <th>Discount_rate</th>\n",
       "      <th>Distance</th>\n",
       "      <th>Date_received</th>\n",
       "      <th>Date</th>\n",
       "      <th>discount_type</th>\n",
       "      <th>discount_rate</th>\n",
       "      <th>discount_man</th>\n",
       "      <th>discount_jian</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1439408</td>\n",
       "      <td>2632</td>\n",
       "      <td>null</td>\n",
       "      <td>null</td>\n",
       "      <td>0</td>\n",
       "      <td>null</td>\n",
       "      <td>20160217</td>\n",
       "      <td>null</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1439408</td>\n",
       "      <td>4663</td>\n",
       "      <td>11002</td>\n",
       "      <td>150:20</td>\n",
       "      <td>1</td>\n",
       "      <td>20160528</td>\n",
       "      <td>null</td>\n",
       "      <td>1</td>\n",
       "      <td>0.866667</td>\n",
       "      <td>150</td>\n",
       "      <td>20</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1439408</td>\n",
       "      <td>2632</td>\n",
       "      <td>8591</td>\n",
       "      <td>20:1</td>\n",
       "      <td>0</td>\n",
       "      <td>20160217</td>\n",
       "      <td>null</td>\n",
       "      <td>1</td>\n",
       "      <td>0.950000</td>\n",
       "      <td>20</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1439408</td>\n",
       "      <td>2632</td>\n",
       "      <td>1078</td>\n",
       "      <td>20:1</td>\n",
       "      <td>0</td>\n",
       "      <td>20160319</td>\n",
       "      <td>null</td>\n",
       "      <td>1</td>\n",
       "      <td>0.950000</td>\n",
       "      <td>20</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1439408</td>\n",
       "      <td>2632</td>\n",
       "      <td>8591</td>\n",
       "      <td>20:1</td>\n",
       "      <td>0</td>\n",
       "      <td>20160613</td>\n",
       "      <td>null</td>\n",
       "      <td>1</td>\n",
       "      <td>0.950000</td>\n",
       "      <td>20</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   User_id  Merchant_id Coupon_id Discount_rate Distance Date_received  \\\n",
       "0  1439408         2632      null          null        0          null   \n",
       "1  1439408         4663     11002        150:20        1      20160528   \n",
       "2  1439408         2632      8591          20:1        0      20160217   \n",
       "3  1439408         2632      1078          20:1        0      20160319   \n",
       "4  1439408         2632      8591          20:1        0      20160613   \n",
       "\n",
       "       Date discount_type  discount_rate  discount_man  discount_jian  \n",
       "0  20160217          null       1.000000             0              0  \n",
       "1      null             1       0.866667           150             20  \n",
       "2      null             1       0.950000            20              1  \n",
       "3      null             1       0.950000            20              1  \n",
       "4      null             1       0.950000            20              1  "
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dfoff.head(5)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 距离 Distance"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Distance 类型： ['0' '1' 'null' '2' '10' '4' '7' '9' '3' '5' '6' '8']\n"
     ]
    }
   ],
   "source": [
    "print('Distance 类型：',dfoff['Distance'].unique())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "将距离 str 转为 int。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[ 0  1 -1  2 10  4  7  9  3  5  6  8]\n",
      "[ 1 -1  5  2  0 10  3  6  7  4  9  8]\n"
     ]
    }
   ],
   "source": [
    "# convert distance\n",
    "dfoff['distance'] = dfoff['Distance'].replace('null', -1).astype(int)\n",
    "print(dfoff['distance'].unique())\n",
    "dftest['distance'] = dftest['Distance'].replace('null', -1).astype(int)\n",
    "print(dftest['distance'].unique())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>User_id</th>\n",
       "      <th>Merchant_id</th>\n",
       "      <th>Coupon_id</th>\n",
       "      <th>Discount_rate</th>\n",
       "      <th>Distance</th>\n",
       "      <th>Date_received</th>\n",
       "      <th>Date</th>\n",
       "      <th>discount_type</th>\n",
       "      <th>discount_rate</th>\n",
       "      <th>discount_man</th>\n",
       "      <th>discount_jian</th>\n",
       "      <th>distance</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1439408</td>\n",
       "      <td>2632</td>\n",
       "      <td>null</td>\n",
       "      <td>null</td>\n",
       "      <td>0</td>\n",
       "      <td>null</td>\n",
       "      <td>20160217</td>\n",
       "      <td>null</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1439408</td>\n",
       "      <td>4663</td>\n",
       "      <td>11002</td>\n",
       "      <td>150:20</td>\n",
       "      <td>1</td>\n",
       "      <td>20160528</td>\n",
       "      <td>null</td>\n",
       "      <td>1</td>\n",
       "      <td>0.866667</td>\n",
       "      <td>150</td>\n",
       "      <td>20</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1439408</td>\n",
       "      <td>2632</td>\n",
       "      <td>8591</td>\n",
       "      <td>20:1</td>\n",
       "      <td>0</td>\n",
       "      <td>20160217</td>\n",
       "      <td>null</td>\n",
       "      <td>1</td>\n",
       "      <td>0.950000</td>\n",
       "      <td>20</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1439408</td>\n",
       "      <td>2632</td>\n",
       "      <td>1078</td>\n",
       "      <td>20:1</td>\n",
       "      <td>0</td>\n",
       "      <td>20160319</td>\n",
       "      <td>null</td>\n",
       "      <td>1</td>\n",
       "      <td>0.950000</td>\n",
       "      <td>20</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1439408</td>\n",
       "      <td>2632</td>\n",
       "      <td>8591</td>\n",
       "      <td>20:1</td>\n",
       "      <td>0</td>\n",
       "      <td>20160613</td>\n",
       "      <td>null</td>\n",
       "      <td>1</td>\n",
       "      <td>0.950000</td>\n",
       "      <td>20</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   User_id  Merchant_id Coupon_id Discount_rate Distance Date_received  \\\n",
       "0  1439408         2632      null          null        0          null   \n",
       "1  1439408         4663     11002        150:20        1      20160528   \n",
       "2  1439408         2632      8591          20:1        0      20160217   \n",
       "3  1439408         2632      1078          20:1        0      20160319   \n",
       "4  1439408         2632      8591          20:1        0      20160613   \n",
       "\n",
       "       Date discount_type  discount_rate  discount_man  discount_jian  \\\n",
       "0  20160217          null       1.000000             0              0   \n",
       "1      null             1       0.866667           150             20   \n",
       "2      null             1       0.950000            20              1   \n",
       "3      null             1       0.950000            20              1   \n",
       "4      null             1       0.950000            20              1   \n",
       "\n",
       "   distance  \n",
       "0         0  \n",
       "1         1  \n",
       "2         0  \n",
       "3         0  \n",
       "4         0  "
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dfoff.head(5)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 领劵日期 Date_received"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "优惠卷收到日期从 20160101 到 20160615\n",
      "消费日期从 20160101 到 20160630\n"
     ]
    }
   ],
   "source": [
    "date_received = dfoff['Date_received'].unique()\n",
    "date_received = sorted(date_received[date_received != 'null'])\n",
    "\n",
    "date_buy = dfoff['Date'].unique()\n",
    "date_buy = sorted(date_buy[date_buy != 'null'])\n",
    "\n",
    "print('优惠卷收到日期从',date_received[0],'到',date_received[-1])\n",
    "print('消费日期从',date_buy[0],'到',date_buy[-1])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**关于领劵日期的特征：**\n",
    "\n",
    "- weekday : {null, 1, 2, 3, 4, 5, 6, 7}\n",
    "\n",
    "- weekday_type : {1, 0}（周六和周日为1，其他为0）\n",
    "\n",
    "- Weekday_1 : {1, 0, 0, 0, 0, 0, 0}\n",
    "\n",
    "- Weekday_2 : {0, 1, 0, 0, 0, 0, 0}\n",
    "\n",
    "- Weekday_3 : {0, 0, 1, 0, 0, 0, 0}\n",
    "\n",
    "- Weekday_4 : {0, 0, 0, 1, 0, 0, 0}\n",
    "\n",
    "- Weekday_5 : {0, 0, 0, 0, 1, 0, 0}\n",
    "\n",
    "- Weekday_6 : {0, 0, 0, 0, 0, 1, 0}\n",
    "\n",
    "- Weekday_7 : {0, 0, 0, 0, 0, 0, 1}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['weekday_1', 'weekday_2', 'weekday_3', 'weekday_4', 'weekday_5', 'weekday_6', 'weekday_7']\n"
     ]
    }
   ],
   "source": [
    "def getWeekday(row):\n",
    "    if row == 'null':\n",
    "        return row\n",
    "    else:\n",
    "        return date(int(row[0:4]), int(row[4:6]), int(row[6:8])).weekday() + 1\n",
    "\n",
    "dfoff['weekday'] = dfoff['Date_received'].astype(str).apply(getWeekday)\n",
    "dftest['weekday'] = dftest['Date_received'].astype(str).apply(getWeekday)\n",
    "\n",
    "# weekday_type :  周六和周日为1，其他为0\n",
    "dfoff['weekday_type'] = dfoff['weekday'].apply(lambda x: 1 if x in [6,7] else 0)\n",
    "dftest['weekday_type'] = dftest['weekday'].apply(lambda x: 1 if x in [6,7] else 0)\n",
    "\n",
    "# change weekday to one-hot encoding \n",
    "weekdaycols = ['weekday_' + str(i) for i in range(1,8)]\n",
    "print(weekdaycols)\n",
    "\n",
    "tmpdf = pd.get_dummies(dfoff['weekday'].replace('null', np.nan))\n",
    "tmpdf.head(10)\n",
    "tmpdf.columns = weekdaycols\n",
    "dfoff[weekdaycols] = tmpdf\n",
    "\n",
    "tmpdf = pd.get_dummies(dftest['weekday'].replace('null', np.nan))\n",
    "tmpdf.columns = weekdaycols\n",
    "dftest[weekdaycols] = tmpdf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>User_id</th>\n",
       "      <th>Merchant_id</th>\n",
       "      <th>Coupon_id</th>\n",
       "      <th>Discount_rate</th>\n",
       "      <th>Distance</th>\n",
       "      <th>Date_received</th>\n",
       "      <th>Date</th>\n",
       "      <th>discount_type</th>\n",
       "      <th>discount_rate</th>\n",
       "      <th>discount_man</th>\n",
       "      <th>...</th>\n",
       "      <th>distance</th>\n",
       "      <th>weekday</th>\n",
       "      <th>weekday_type</th>\n",
       "      <th>weekday_1</th>\n",
       "      <th>weekday_2</th>\n",
       "      <th>weekday_3</th>\n",
       "      <th>weekday_4</th>\n",
       "      <th>weekday_5</th>\n",
       "      <th>weekday_6</th>\n",
       "      <th>weekday_7</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1439408</td>\n",
       "      <td>2632</td>\n",
       "      <td>null</td>\n",
       "      <td>null</td>\n",
       "      <td>0</td>\n",
       "      <td>null</td>\n",
       "      <td>20160217</td>\n",
       "      <td>null</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>null</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1439408</td>\n",
       "      <td>4663</td>\n",
       "      <td>11002</td>\n",
       "      <td>150:20</td>\n",
       "      <td>1</td>\n",
       "      <td>20160528</td>\n",
       "      <td>null</td>\n",
       "      <td>1</td>\n",
       "      <td>0.866667</td>\n",
       "      <td>150</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>6</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1439408</td>\n",
       "      <td>2632</td>\n",
       "      <td>8591</td>\n",
       "      <td>20:1</td>\n",
       "      <td>0</td>\n",
       "      <td>20160217</td>\n",
       "      <td>null</td>\n",
       "      <td>1</td>\n",
       "      <td>0.950000</td>\n",
       "      <td>20</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1439408</td>\n",
       "      <td>2632</td>\n",
       "      <td>1078</td>\n",
       "      <td>20:1</td>\n",
       "      <td>0</td>\n",
       "      <td>20160319</td>\n",
       "      <td>null</td>\n",
       "      <td>1</td>\n",
       "      <td>0.950000</td>\n",
       "      <td>20</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>6</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1439408</td>\n",
       "      <td>2632</td>\n",
       "      <td>8591</td>\n",
       "      <td>20:1</td>\n",
       "      <td>0</td>\n",
       "      <td>20160613</td>\n",
       "      <td>null</td>\n",
       "      <td>1</td>\n",
       "      <td>0.950000</td>\n",
       "      <td>20</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 21 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   User_id  Merchant_id Coupon_id Discount_rate Distance Date_received  \\\n",
       "0  1439408         2632      null          null        0          null   \n",
       "1  1439408         4663     11002        150:20        1      20160528   \n",
       "2  1439408         2632      8591          20:1        0      20160217   \n",
       "3  1439408         2632      1078          20:1        0      20160319   \n",
       "4  1439408         2632      8591          20:1        0      20160613   \n",
       "\n",
       "       Date discount_type  discount_rate  discount_man    ...      distance  \\\n",
       "0  20160217          null       1.000000             0    ...             0   \n",
       "1      null             1       0.866667           150    ...             1   \n",
       "2      null             1       0.950000            20    ...             0   \n",
       "3      null             1       0.950000            20    ...             0   \n",
       "4      null             1       0.950000            20    ...             0   \n",
       "\n",
       "   weekday weekday_type  weekday_1  weekday_2  weekday_3  weekday_4  \\\n",
       "0     null            0          0          0          0          0   \n",
       "1        6            1          0          0          0          0   \n",
       "2        3            0          0          0          1          0   \n",
       "3        6            1          0          0          0          0   \n",
       "4        1            0          1          0          0          0   \n",
       "\n",
       "   weekday_5  weekday_6  weekday_7  \n",
       "0          0          0          0  \n",
       "1          0          1          0  \n",
       "2          0          0          0  \n",
       "3          0          1          0  \n",
       "4          0          0          0  \n",
       "\n",
       "[5 rows x 21 columns]"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dfoff.head(5)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 所有特征："
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- discount_rate\n",
    "\n",
    "- discount_type\n",
    "\n",
    "- discount_man\n",
    "\n",
    "- discount_jian\n",
    "\n",
    "- distance\n",
    "\n",
    "- weekday\n",
    "\n",
    "- weekday_type\n",
    "\n",
    "- weekday_1\n",
    "\n",
    "- weekday_2\n",
    "\n",
    "- weekday_3\n",
    "\n",
    "- weekday_4\n",
    "\n",
    "- weekday_5\n",
    "\n",
    "- weekday_6\n",
    "\n",
    "- weekday_7"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 标签标注"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "三种情况：\n",
    "\n",
    "- Date_received == 'null'：表示没有领到优惠券，无需考虑，y = -1\n",
    "\n",
    "- (Date_received != 'null') & (Date != 'null') & (Date - Date_received <= 15)：表示领取优惠券且在15天内使用，即正样本，y = 1\n",
    "\n",
    "- (Date_received != 'null') & ((Date == 'null') | (Date - Date_received > 15))：表示领取优惠券未在在15天内使用，即负样本，y = 0"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "定义标签备注函数："
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def label(row):\n",
    "    if row['Date_received'] == 'null':\n",
    "        return -1\n",
    "    if row['Date'] != 'null':\n",
    "        td = pd.to_datetime(row['Date'], format='%Y%m%d') - pd.to_datetime(row['Date_received'], format='%Y%m%d')\n",
    "        if td <= pd.Timedelta(15, 'D'):\n",
    "            return 1\n",
    "    return 0\n",
    "\n",
    "dfoff['label'] = dfoff.apply(label, axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " 0    988887\n",
      "-1    701602\n",
      " 1     64395\n",
      "Name: label, dtype: int64\n"
     ]
    }
   ],
   "source": [
    "print(dfoff['label'].value_counts())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>User_id</th>\n",
       "      <th>Merchant_id</th>\n",
       "      <th>Coupon_id</th>\n",
       "      <th>Discount_rate</th>\n",
       "      <th>Distance</th>\n",
       "      <th>Date_received</th>\n",
       "      <th>Date</th>\n",
       "      <th>discount_type</th>\n",
       "      <th>discount_rate</th>\n",
       "      <th>discount_man</th>\n",
       "      <th>...</th>\n",
       "      <th>weekday</th>\n",
       "      <th>weekday_type</th>\n",
       "      <th>weekday_1</th>\n",
       "      <th>weekday_2</th>\n",
       "      <th>weekday_3</th>\n",
       "      <th>weekday_4</th>\n",
       "      <th>weekday_5</th>\n",
       "      <th>weekday_6</th>\n",
       "      <th>weekday_7</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1439408</td>\n",
       "      <td>2632</td>\n",
       "      <td>null</td>\n",
       "      <td>null</td>\n",
       "      <td>0</td>\n",
       "      <td>null</td>\n",
       "      <td>20160217</td>\n",
       "      <td>null</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>null</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1439408</td>\n",
       "      <td>4663</td>\n",
       "      <td>11002</td>\n",
       "      <td>150:20</td>\n",
       "      <td>1</td>\n",
       "      <td>20160528</td>\n",
       "      <td>null</td>\n",
       "      <td>1</td>\n",
       "      <td>0.866667</td>\n",
       "      <td>150</td>\n",
       "      <td>...</td>\n",
       "      <td>6</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1439408</td>\n",
       "      <td>2632</td>\n",
       "      <td>8591</td>\n",
       "      <td>20:1</td>\n",
       "      <td>0</td>\n",
       "      <td>20160217</td>\n",
       "      <td>null</td>\n",
       "      <td>1</td>\n",
       "      <td>0.950000</td>\n",
       "      <td>20</td>\n",
       "      <td>...</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1439408</td>\n",
       "      <td>2632</td>\n",
       "      <td>1078</td>\n",
       "      <td>20:1</td>\n",
       "      <td>0</td>\n",
       "      <td>20160319</td>\n",
       "      <td>null</td>\n",
       "      <td>1</td>\n",
       "      <td>0.950000</td>\n",
       "      <td>20</td>\n",
       "      <td>...</td>\n",
       "      <td>6</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1439408</td>\n",
       "      <td>2632</td>\n",
       "      <td>8591</td>\n",
       "      <td>20:1</td>\n",
       "      <td>0</td>\n",
       "      <td>20160613</td>\n",
       "      <td>null</td>\n",
       "      <td>1</td>\n",
       "      <td>0.950000</td>\n",
       "      <td>20</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 22 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   User_id  Merchant_id Coupon_id Discount_rate Distance Date_received  \\\n",
       "0  1439408         2632      null          null        0          null   \n",
       "1  1439408         4663     11002        150:20        1      20160528   \n",
       "2  1439408         2632      8591          20:1        0      20160217   \n",
       "3  1439408         2632      1078          20:1        0      20160319   \n",
       "4  1439408         2632      8591          20:1        0      20160613   \n",
       "\n",
       "       Date discount_type  discount_rate  discount_man  ...    weekday  \\\n",
       "0  20160217          null       1.000000             0  ...       null   \n",
       "1      null             1       0.866667           150  ...          6   \n",
       "2      null             1       0.950000            20  ...          3   \n",
       "3      null             1       0.950000            20  ...          6   \n",
       "4      null             1       0.950000            20  ...          1   \n",
       "\n",
       "   weekday_type weekday_1  weekday_2  weekday_3  weekday_4  weekday_5  \\\n",
       "0             0         0          0          0          0          0   \n",
       "1             1         0          0          0          0          0   \n",
       "2             0         0          0          1          0          0   \n",
       "3             1         0          0          0          0          0   \n",
       "4             0         1          0          0          0          0   \n",
       "\n",
       "   weekday_6  weekday_7  label  \n",
       "0          0          0     -1  \n",
       "1          1          0      0  \n",
       "2          0          0      0  \n",
       "3          1          0      0  \n",
       "4          0          0      0  \n",
       "\n",
       "[5 rows x 22 columns]"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dfoff.head(5)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 建立线性模型 SGDClassifier"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- 使用上面提取的14个特征。\n",
    "\n",
    "- 训练集：20160101-20160515；验证集：20160516-20160615。\n",
    "\n",
    "- 用线性模型 SGDClassifier"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 划分训练集/验证集"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train Set: \n",
      " 0    759172\n",
      "1     41524\n",
      "Name: label, dtype: int64\n",
      "Valid Set: \n",
      " 0    229715\n",
      "1     22871\n",
      "Name: label, dtype: int64\n"
     ]
    }
   ],
   "source": [
    "# data split\n",
    "df = dfoff[dfoff['label'] != -1].copy()\n",
    "train = df[(df['Date_received'] < '20160516')].copy()\n",
    "valid = df[(df['Date_received'] >= '20160516') & (df['Date_received'] <= '20160615')].copy()\n",
    "print('Train Set: \\n', train['label'].value_counts())\n",
    "print('Valid Set: \\n', valid['label'].value_counts())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 特征数量"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "共有特征： 14 个\n",
      "['discount_rate', 'discount_type', 'discount_man', 'discount_jian', 'distance', 'weekday', 'weekday_type', 'weekday_1', 'weekday_2', 'weekday_3', 'weekday_4', 'weekday_5', 'weekday_6', 'weekday_7']\n"
     ]
    }
   ],
   "source": [
    "# feature\n",
    "original_feature = ['discount_rate','discount_type','discount_man', 'discount_jian','distance', 'weekday', 'weekday_type'] + weekdaycols\n",
    "print('共有特征：',len(original_feature),'个')\n",
    "print(original_feature)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 建立模型"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.metrics import accuracy_score         # 用来计算 XGB 的预测准确率\n",
    "import xgboost as xgb\n",
    "def check_model(data, predictors):\n",
    "    params={'objective':'multi:softmax',            # 定义多分类问题\n",
    "       'num_class':10,                          # 类别个数\n",
    "       'eta':0.1,                              # 学习率\n",
    "       'silent':1                              # 是否打印中间结果，1就是不打印\n",
    "       }\n",
    "    train = xgb.DMatrix(data[predictors],data['label'])\n",
    "    num_round = 5\n",
    "    bst = xgb.train(params,train,num_round)\n",
    "\n",
    "    return grid_search"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 训练"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [
    {
     "ename": "AttributeError",
     "evalue": "type object 'XGBClassifier' has no attribute 'DMatrix'",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mAttributeError\u001b[0m                            Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-69-d16d91839f2e>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[0;32m      1\u001b[0m \u001b[0mpredictors\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0moriginal_feature\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 2\u001b[1;33m \u001b[0mmodel\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mcheck_model\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mtrain\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mpredictors\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[1;32m<ipython-input-68-b11efde1f3a3>\u001b[0m in \u001b[0;36mcheck_model\u001b[1;34m(data, predictors)\u001b[0m\n\u001b[0;32m      7\u001b[0m        \u001b[1;34m'silent'\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;36m1\u001b[0m                              \u001b[1;31m# 是否打印中间结果，1就是不打印\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      8\u001b[0m        }\n\u001b[1;32m----> 9\u001b[1;33m     \u001b[0mtrain\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mXGBClassifier\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mDMatrix\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mdata\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mpredictors\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mdata\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m'label'\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     10\u001b[0m     \u001b[0mnum_round\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;36m5\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     11\u001b[0m     \u001b[0mbst\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mxgboost\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mtrain\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mparams\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mtrain\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mnum_round\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mAttributeError\u001b[0m: type object 'XGBClassifier' has no attribute 'DMatrix'"
     ]
    }
   ],
   "source": [
    "predictors = original_feature\n",
    "model = check_model(train, predictors)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 验证"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "对验证集中每个优惠券预测的结果计算 AUC，再对所有优惠券的 AUC 求平均。计算 AUC 的时候，如果 label 只有一类，就直接跳过，因为 AUC 无法计算。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>User_id</th>\n",
       "      <th>Merchant_id</th>\n",
       "      <th>Coupon_id</th>\n",
       "      <th>Discount_rate</th>\n",
       "      <th>Distance</th>\n",
       "      <th>Date_received</th>\n",
       "      <th>Date</th>\n",
       "      <th>discount_type</th>\n",
       "      <th>discount_rate</th>\n",
       "      <th>discount_man</th>\n",
       "      <th>...</th>\n",
       "      <th>weekday_type</th>\n",
       "      <th>weekday_1</th>\n",
       "      <th>weekday_2</th>\n",
       "      <th>weekday_3</th>\n",
       "      <th>weekday_4</th>\n",
       "      <th>weekday_5</th>\n",
       "      <th>weekday_6</th>\n",
       "      <th>weekday_7</th>\n",
       "      <th>label</th>\n",
       "      <th>pred_prob</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1439408</td>\n",
       "      <td>4663</td>\n",
       "      <td>11002</td>\n",
       "      <td>150:20</td>\n",
       "      <td>1</td>\n",
       "      <td>20160528</td>\n",
       "      <td>null</td>\n",
       "      <td>1</td>\n",
       "      <td>0.866667</td>\n",
       "      <td>150</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.019497</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1439408</td>\n",
       "      <td>2632</td>\n",
       "      <td>8591</td>\n",
       "      <td>20:1</td>\n",
       "      <td>0</td>\n",
       "      <td>20160613</td>\n",
       "      <td>null</td>\n",
       "      <td>1</td>\n",
       "      <td>0.950000</td>\n",
       "      <td>20</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.101397</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>1439408</td>\n",
       "      <td>2632</td>\n",
       "      <td>8591</td>\n",
       "      <td>20:1</td>\n",
       "      <td>0</td>\n",
       "      <td>20160516</td>\n",
       "      <td>20160613</td>\n",
       "      <td>1</td>\n",
       "      <td>0.950000</td>\n",
       "      <td>20</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.101397</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>2029232</td>\n",
       "      <td>450</td>\n",
       "      <td>1532</td>\n",
       "      <td>30:5</td>\n",
       "      <td>0</td>\n",
       "      <td>20160530</td>\n",
       "      <td>null</td>\n",
       "      <td>1</td>\n",
       "      <td>0.833333</td>\n",
       "      <td>30</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.097119</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>2029232</td>\n",
       "      <td>6459</td>\n",
       "      <td>12737</td>\n",
       "      <td>20:1</td>\n",
       "      <td>0</td>\n",
       "      <td>20160519</td>\n",
       "      <td>null</td>\n",
       "      <td>1</td>\n",
       "      <td>0.950000</td>\n",
       "      <td>20</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.132886</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 23 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "    User_id  Merchant_id Coupon_id Discount_rate Distance Date_received  \\\n",
       "1   1439408         4663     11002        150:20        1      20160528   \n",
       "4   1439408         2632      8591          20:1        0      20160613   \n",
       "6   1439408         2632      8591          20:1        0      20160516   \n",
       "9   2029232          450      1532          30:5        0      20160530   \n",
       "10  2029232         6459     12737          20:1        0      20160519   \n",
       "\n",
       "        Date discount_type  discount_rate  discount_man    ...      \\\n",
       "1       null             1       0.866667           150    ...       \n",
       "4       null             1       0.950000            20    ...       \n",
       "6   20160613             1       0.950000            20    ...       \n",
       "9       null             1       0.833333            30    ...       \n",
       "10      null             1       0.950000            20    ...       \n",
       "\n",
       "    weekday_type  weekday_1 weekday_2  weekday_3  weekday_4  weekday_5  \\\n",
       "1              1          0         0          0          0          0   \n",
       "4              0          1         0          0          0          0   \n",
       "6              0          1         0          0          0          0   \n",
       "9              0          1         0          0          0          0   \n",
       "10             0          0         0          0          1          0   \n",
       "\n",
       "    weekday_6  weekday_7  label  pred_prob  \n",
       "1           1          0      0   0.019497  \n",
       "4           0          0      0   0.101397  \n",
       "6           0          0      0   0.101397  \n",
       "9           0          0      0   0.097119  \n",
       "10          0          0      0   0.132886  \n",
       "\n",
       "[5 rows x 23 columns]"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# valid predict\n",
    "y_valid_pred = model.predict_proba(valid[predictors])\n",
    "valid1 = valid.copy()\n",
    "valid1['pred_prob'] = y_valid_pred[:, 1]\n",
    "valid1.head(5)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "计算 AUC："
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.532344469452\n"
     ]
    }
   ],
   "source": [
    "# avgAUC calculation\n",
    "vg = valid1.groupby(['Coupon_id'])\n",
    "aucs = []\n",
    "for i in vg:\n",
    "    tmpdf = i[1] \n",
    "    if len(tmpdf['label'].unique()) != 2:\n",
    "        continue\n",
    "    fpr, tpr, thresholds = roc_curve(tmpdf['label'], tmpdf['pred_prob'], pos_label=1)\n",
    "    aucs.append(auc(fpr, tpr))\n",
    "print(np.average(aucs))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 测试"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>User_id</th>\n",
       "      <th>Coupon_id</th>\n",
       "      <th>Date_received</th>\n",
       "      <th>Probability</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>4129537</td>\n",
       "      <td>9983</td>\n",
       "      <td>20160712</td>\n",
       "      <td>0.105409</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>6949378</td>\n",
       "      <td>3429</td>\n",
       "      <td>20160706</td>\n",
       "      <td>0.153780</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2166529</td>\n",
       "      <td>6928</td>\n",
       "      <td>20160727</td>\n",
       "      <td>0.005531</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2166529</td>\n",
       "      <td>1808</td>\n",
       "      <td>20160727</td>\n",
       "      <td>0.018747</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>6172162</td>\n",
       "      <td>6500</td>\n",
       "      <td>20160708</td>\n",
       "      <td>0.063571</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   User_id  Coupon_id  Date_received  Probability\n",
       "0  4129537       9983       20160712     0.105409\n",
       "1  6949378       3429       20160706     0.153780\n",
       "2  2166529       6928       20160727     0.005531\n",
       "3  2166529       1808       20160727     0.018747\n",
       "4  6172162       6500       20160708     0.063571"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# test prediction for submission\n",
    "y_test_pred = model.predict_proba(dftest[predictors])\n",
    "dftest1 = dftest[['User_id','Coupon_id','Date_received']].copy()\n",
    "dftest1['Probability'] = y_test_pred[:,1]\n",
    "dftest1.to_csv('submit1.csv', index=False, header=False)\n",
    "dftest1.head(5)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 保存模型 & 导入模型"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "if not os.path.isfile('1_model.pkl'):\n",
    "    with open('1_model.pkl', 'wb') as f:\n",
    "        pickle.dump(model, f)\n",
    "else:\n",
    "    with open('1_model.pkl', 'rb') as f:\n",
    "        model = pickle.load(f)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 优化模型..."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- **特征工程**\n",
    "\n",
    "- **机器学习算法**\n",
    "\n",
    "- **模型集成**"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 参考代码"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "[比赛第一名代码与解析](https://github.com/wepe/O2O-Coupon-Usage-Forecast)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
